import re
from scrapy.http import Request
import util

class SinaDownloadMiddleware(object):

    black_pattern = re.compile(r"\r|\n|\t")

    def process_request(self, request, spider):
        request.headers[b"User-Agent"] = util.getRandomUserAgent()
        # request.headers.setdefault(b'referer', 'http://roll.news.sina.com.cn')
        request.headers[b'referer'] = 'http://roll.news.sina.com.cn'
        return None

    def process_response(self, request, response, spider):
        if "roll.news.sina.com.cn" not in request.url:
            temp = re.sub(self.black_pattern, "", response.text)
            temp = temp.replace(u"\u3000", "").replace(u"\xa0", "")
            response = response.replace(body=temp)
        return response
    
class SinaMiddleware(object):
        
    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

    # def process_start_requests(self, start_requests, spider):
    #     for r in start_requests:
    #         if not spider.date:
    #             spider.date = time.strftime('%Y-%m-%d',time.localtime(time.time()-86400))
    #             url = spider.ajax_url + "&page=1&date=" + spider.date
    #             r = Request(url)
    #         yield r
